package com.solidtrust.crawlers;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.solidtrust.crawler.framework.DistributedSitemap;
import com.solidtrust.crawler.framework.NotSameCityException;
import com.solidtrust.crawler.model.PageSource;
import com.solidtrust.crawler.service.GanjiService;
import com.solidtrust.entity.Ganji;
import com.solidtrust.entity.IScrapeEntity;
import com.solidtrust.util.DateUtil;
import com.solidtrust.util.HttpUtil;
import com.solidtrust.util.ScrapeUtil;

@Service
public class GanjiSitemap extends DistributedSitemap{
	private static final Logger log4j = Logger.getLogger(GanjiSitemap.class);
	private static final Date init = new Date(0);
	private Map<String, String> modelMapping;
	private HttpClient httpClient = HttpUtil.getHttpClient();
	private int repeat = 0;
	private int interval = 1000;
	
	@Autowired GanjiService svc;
	
	@Override
	public void start() throws Exception {
		modelMapping = svc.getModelMap();
		super.start();
	}
	

	@Override
	public String getSource() {
		return "ganji";
	}


	@Override
	protected Class<? extends IScrapeEntity> getEntityClass() {
		return Ganji.class;
	}

	@Override
	protected String getRowElementSelector() {
		return ".layoutlist > .list-pic";
	}

	@Override
	protected IScrapeEntity eachItem(Element ele, String model, String brand) throws IOException {
		String href = ele.select("a").first().attr("abs:href");
		if(!isSameCity(href)){
			log4j.info("Different city: " + href);
			throw new NotSameCityException(href);
		}
		String carId = null;
		String puid = ele.attr("id");
		if(StringUtils.isNotEmpty(puid)){
			carId = puid.substring(5);
		}else{
			int i = href.indexOf("x.htm");
			if(i > -1){
				int y = href.lastIndexOf("/", i);
				if(y > -1){
					carId = href.substring(y + 1, i);
				}
			}	
		}
		if(carId == null){
			log4j.info("Invalid id: " + href);
			return null;
		}
		log4j.info("Collect car with ID " + carId + " from " + href);
		String title = ele.select("a.infor-title").text();
		if(title == null || !title.replace(" ", "").contains(brand)){
			log4j.info("Invalid ad: " + title);
			return null;
		}
		boolean notSold = true;
		if(checkExist(carId, !notSold) == false){
			return null;
		}
		
		Ganji ganji = new Ganji();
		ganji.setSourceId(carId);
		ganji.setAdUrl(href);
		ganji.setAdTitle(title);
		ganji.setBuild(ScrapeUtil.getBuild(title));
		
		try {
			String age = ele.select(".js-license > strong").text();
			Date d = DateUtils.addYears(new Date(), 0 - Integer.parseInt(age));
			d = DateUtils.truncate(d, Calendar.YEAR);
			ganji.setBuyDate(d);
		} catch (Exception e) {}
		
		try {
			String text = ele.select(".js-roadHaul > strong").text();
			if(text.startsWith("<")){
				ganji.setMileage(ScrapeUtil.getWan(text.substring(1)) - 1);
			}else{
				ganji.setMileage(ScrapeUtil.getWan(text));
			}
		} catch (Exception e) {}
		
		try {
			String text = ele.select(".v-Price > .js-price").text();
			ganji.setSoldPrice(ScrapeUtil.getWan(text));
		} catch (Exception e) {}
		
		try {
			boolean notBang = ele.select(".ico-bang-new").isEmpty();
			if(notBang){
				String text = ele.select(".infor-gs > .fc-gray").first().text();
				String type = text.substring(1, text.length() - 1);
				ganji.setBuyerType(type);
				
				if(type.equals("商家")){
					text = ele.select(".infor-gs > .fc-gray").last().text();
					ganji.setSellerCompany(text);
				}
			}else{
				String text = ele.select(".infor-gs > a.fc-999").text();
				ganji.setSellerCompany(text);
				ganji.setBuyerType("商家");
			}
			String district = ele.select(".infor-gs > .list-word").text();
			ganji.setDistrict(district);
		} catch (Exception e) {}
		
		try {
			String text = ele.select(".infor-gs > .list-word").text();
			if(ganji.getBuyerType().equals("个人")){
				ganji.setDistrict(text);
			}else{
				ganji.setSellerCompany(text);
			}
		} catch (Exception e) {}
		
		PageSource page = new PageSource();
		page.setName(carId);
		page.setUrl(href);
		try {
			eachDetailPage(page, ganji);
		} catch (Exception e) {
			log4j.info(e.getMessage());
		}
		
		return ganji;
	}
	
	private void eachDetailPage(PageSource page, Ganji ganji) throws Exception {
		Document doc = parseURL(page.getUrl());
		
		Element leftBox = doc.select(".leftBox").first();
		if(leftBox == null){ //iframe
			Element iframe = doc.select("#Iframe32").first();
			if(iframe != null){
				ganji.setSourceURL(iframe.attr("src"));
				ganji.setRemovedDate(init);
			}
			return;
		}
		
		String modelUrl = doc.select(".h-search .crumbs > a").last().attr("href");
		String model = modelMapping.get(modelUrl);
		if(model != null){
			String exactModel = svc.getExactModel(ganji.getAdTitle(), model);
			if(!model.equals(exactModel)){
				log4j.info(model + "=>" + exactModel);
			}
			ganji.setModel(model);
		}
		
		String title = leftBox.select(".title-name").first().text().trim();
		ganji.setBuild(ScrapeUtil.getBuild(title));
		
		Element basic = leftBox.select(".basic-info").first();
		try {
			int price = ScrapeUtil.getWan(basic.select(".iNew-price .shengqian-pop .fc-org > i").first().text());
			ganji.setNewPrice(price);
		} catch (Exception e) {}
		try {
			String date = basic.select(".iNew-yeah > div > i").text();
			ganji.setBuyDate(DateUtil.string2Date4(date));
		} catch (Exception e) {}
		
		try {
			String text = basic.select(".iNew-hu > span > i").text();
			ganji.setSoldTimes(Integer.parseInt(text.replace("次", "")));
		} catch (Exception e) {}
		
		Element contact = leftBox.select(".veh-tel").first();
		ganji.setPhone(contact.select(".telephone").text());
		
		//personal phone, need one more http request
//		if(StringUtils.isEmpty(ganji.getPhone())){
//			String wap = contact.select(".canvasimg").attr("data-content");
//			if(wap.startsWith("http")){
//				Document d = parseURL(wap);
//				Elements eles = d.select(".detail-describe p");
//				for (int i = 0; i < eles.size(); i++) {
//					String s = eles.get(i).text();
//					String value = ScrapeUtil.getString(s, "电话");
//					if(value != null){
//						ganji.setPhone(value);
//						break;
//					}
//				}
//			}
//		}
		
		try{
			String seller = contact.select("p.v-p2").first().childNode(1).toString().trim();
			ganji.setSellerName(ScrapeUtil.deleteBracket(seller));
		} catch (Exception e) {}
		
		if(ganji.getBuyerType().equals("商家")){
			try{
				String sellerCompany = doc.select(".rightBar .certdl-det p").text().trim();
				ganji.setSellerCompany(sellerCompany);
				String sellerAddr = doc.select(".rightBar .vc-dealer p").text().trim();
				ganji.setSellerAddress(sellerAddr);
			} catch (Exception e) {}
		}
		
		Element m40 = doc.select(".mt40").first();
		if(m40 == null){
			m40 = doc.select(".mt20").first();
		}
		String text = m40.select(".pz-param-title").text();
		if(StringUtils.isNotBlank(text)){
			ganji.setType(text);
		}else{
			String adtitle = ganji.getAdTitle();
			int index = adtitle.indexOf("款");
			ganji.setType(adtitle.substring(index + 1).trim());
		}
		
		Elements params = m40.select(".det-basinfor li");
		for (int i = 0; i < params.size(); i++) {
			String s = params.get(i).text();
			String value = null;
			if((value = ScrapeUtil.getString(s, "年检到期")) != null){
				try {
					ganji.setInspectionDate(DateUtil.parseMonth(value));
				} catch (Exception e) {}
			}
			else if((value = ScrapeUtil.getString(s, "车身颜色")) != null){
				ganji.setColor(value);
			}
			else if((value = ScrapeUtil.getString(s, "排放标准")) != null){
				int x = value.indexOf(" ");
				if(x != -1){
					value = value.substring(0, x);
				}
				ganji.setEmission(value);
			}
			else if((value = ScrapeUtil.getString(s, "排 气 量")) != null){
				try {
					ganji.setVolume((int)(Double.parseDouble(value.replace("L", "")) * 1000));
				} catch (Exception e) {}
			}
		}
		
		Elements paramList = m40.select(".sim-tl-list li");
		for (int i = 0; i < paramList.size(); i++) {
			Element item = paramList.get(i);
			String s = item.text();
			if("燃油种类".equals(s)){
				ganji.setFuelType(item.nextElementSibling().text());
				break;
			}
		}
		
	}

	@Override
	protected PageSource getNextPage(Document doc) {
		Element next = doc.select(".leftBox a.next").first();
		if(next != null){
			PageSource nextPage = new PageSource();
			nextPage.setUrl(next.attr("abs:href"));
			return nextPage;
		}
		return null;
	}
	
	public void setInterval(int interval) {
		this.interval = interval;
	}
	
	@Override
	protected Document parseURL(String url) throws Exception{
		log4j.info("Scrape page:" + url);
		try {
			Thread.sleep(interval);
		} catch (InterruptedException e1) {}
		totalRequests++;
		
		URI uri = new URI(url);
		HttpGet request = new HttpGet(uri);
		HttpUtil.addHttpHeads(request, uri.getHost());
		InputStream in = null;
		Document doc = null;
		try{
			HttpResponse response = httpClient.execute(request);
			in = response.getEntity().getContent();
			doc = Jsoup.parse(in, "utf-8", url);
			if(doc.select("#checkcodeerror").isEmpty()){
				repeat = 0;
				return doc;
			}
		}catch(Exception e){
			log4j.info(e.getMessage(), e);
			repeat ++;
			log4j.info("repeat=" + repeat);
			if(repeat < 3){
				try {
					Thread.sleep(1000 * 60 * 10);
				} catch (InterruptedException e1) {}
//				log4j.info("----------->refresh cookie");
//				HttpUtil.refreshCookie();
				return parseURL(url);
			}else{
				return null;
			}
		}finally{
			IOUtils.closeQuietly(in);
		}
		
		log4j.info("Total requests: " + totalRequests);
		log4j.error("Blocked!!! Retry 1 minutes later.");
		persist();
		try {
			Thread.sleep(1000 * 60 * 30);
		} catch (InterruptedException e) {}
//		log4j.info("----------->refresh cookie");
//		HttpUtil.refreshCookie();
		return parseURL(url);
	}

	
	public static void main(String[] args) throws Exception {
		GanjiSitemap ganji = new GanjiSitemap();
		String url = ("http://wx.ganji.com/ershouche/28343001909300x.htm");
		PageSource p = new PageSource();
		p.setUrl(url);
		Ganji g = new Ganji();
		g.setBuyerType("商家");
		ganji.eachDetailPage(p, g);
	}
}
